/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package cn.ac.iie.di.postbar.handler.mpost;

import LangTools.LangIdentifyTool;
import cn.ac.iie.di.ban.ycl.idgen.IDGeneratorInterface;
import cn.ac.iie.di.ban.ycl.idgen.IDSHAGeneratorImpl;
import cn.ac.iie.di.datadock.rdata.exchange.client.v1.session.RESendSession;
import cn.ac.iie.di.postbar.entity.MPostEntity;
import cn.ac.iie.di.postbar.handler.mpost.MPostHandler.SessionPool;
import static cn.ac.iie.di.postbar.server.PostLtStartUp.isTianYa;
import static cn.ac.iie.di.postbar.server.PostLtStartUp.requestServerSize;
import static cn.ac.iie.di.postbar.server.PostLtStartUp.userMongoCollection;
import static cn.ac.iie.di.postbar.server.PostLtStartUp.yunrunURL;
import cn.ac.iie.di.postbar.util.Base64;
import cn.ac.iie.di.postbar.util.DeDuplication;
import cn.ac.iie.di.postbar.util.MD5Util;
import cn.ac.iie.di.postbar.util.MSiteLocation;
import cn.ac.iie.di.postbar.util.ParserJson;
import cn.ac.iie.di.postbar.util.SearchMongodb;
import cn.ac.iie.di.postbar.util.TRSTools;
import cn.ac.iie.di.postbar.util.TransformUtil;
import com.alibaba.fastjson.JSONObject;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.sf.json.JSONArray;

import org.apache.log4j.Logger;

/**
 *
 * @author Administrator
 */
public class MPostThreadPoolTask implements Runnable {

    static private final Logger logger = Logger.getLogger(MPostThreadPoolTask.class.getName());
    private final List<String> jsonList;
    private final int timeout;
    private final SessionPool pool;
    private long starttime = System.currentTimeMillis() / 1000;

    public MPostThreadPoolTask(List<String> jsonList, SessionPool pool, int timeout) {
        this.jsonList = jsonList;
        this.pool = pool;
        this.timeout = timeout;
    }

    @Override
    public void run() {
        RESendSession session = null;
        TRSTools trs = new TRSTools();
        SearchMongodb searchmongodb = new SearchMongodb();
        //计数
        long count = 0;
        //请求去重服务字符串
        String requestString = "";
        //存放mpost对象
        List<MPostEntity> mpostEntityList = new ArrayList<MPostEntity>();

        try {
            session = pool.borrowSession();
            for (String json : jsonList) {
                if (json != null && !json.isEmpty()) {
                    //解析json
                    List<Map<String, String>> dataMapList = new ArrayList<Map<String, String>>();
                    try {
                        dataMapList = ParserJson.getContentByRecord_list(json);
                        logger.debug("parser json data successfully.");
                    } catch (Exception e) {
                        logger.error("@@@ERROR_MESSAGE@@@:" + json);
                        logger.error("parser data error,cause " + e.getMessage(), e);
                        return;
                    }
                    logger.info("the json has " + dataMapList.size() + " data.");
                    for (int i = 0; i < dataMapList.size(); i++) {
                        MPostEntity mpostEntity = new MPostEntity();
                        Map<String, String> dataMap = new HashMap<String, String>();
                        dataMap = dataMapList.get(i);

                        IDGeneratorInterface idge = new IDSHAGeneratorImpl();
//
//                                                mpostEntity.setG_channel(2);
//                                                mpostEntity.setG_spec("M-BBS01-AI");

                        String m_url = dataMap.get("ORIGINURL");
                        if (m_url == null || m_url.isEmpty() || "null".equals(m_url)) {
                            mpostEntity.setM_url("");
                        } else {
                            try {
                                m_url = new String(Base64.decodeFast(m_url), "UTF-8");
                                mpostEntity.setM_url(m_url);
                            } catch (Exception e) {
                                logger.error("(m_url)Decode base64 failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_url:" + dataMap.get("ORIGINURL"));
                                mpostEntity.setM_url("");
                                m_url = "";
                            }
                        }

                        String u_access_type = dataMap.get("CHANNEL");
                        long uat = -1;
                        if (u_access_type == null || u_access_type.isEmpty() || "null".equals(u_access_type)) {
                            mpostEntity.setU_access_type(-1);
                        } else {
                            try {
                                uat = Long.parseLong(u_access_type);
                                mpostEntity.setU_access_type(uat);
                            } catch (Exception e) {
                                logger.error("(u_access_type)Type conversion failed, cause " + e.getMessage(), e);
                                logger.warn("@@u_access_type:" + u_access_type);
                                mpostEntity.setU_access_type(-1);
                                uat = -1;
                            }
                        }

                        String g_ch_key = m_url;
                        mpostEntity.setG_ch_key(g_ch_key);

//                        if (isTianYa == 1) {
//                            mpostEntity.setG_asp("tianya.cn");
//                            mpostEntity.setG_adp("tianya.cn");
//                        } else {
//                            mpostEntity.setG_asp("kdnet.net");
//                            mpostEntity.setG_adp("kdnet.net");
//                        }
                        if (isTianYa == 1) {
                            mpostEntity.setG_asp("tianya.cn");
                            mpostEntity.setG_adp("tianya.cn");
                        } else if (isTianYa == 2) {//人民网强国论坛
                            mpostEntity.setG_asp("people.com.cn");
                            mpostEntity.setG_adp("people.com.cn");
                        } else if (isTianYa == 3) {//爱卡
                            mpostEntity.setG_asp("xcar.com.cn");
                            mpostEntity.setG_adp("xcar.com.cn");
                        } else if (isTianYa == 4) {//铁血
                            mpostEntity.setG_asp("tiexue.net");
                            mpostEntity.setG_adp("tiexue.net");
                        } else if (isTianYa == 5) {//中华
                            mpostEntity.setG_asp("china.com");
                            mpostEntity.setG_adp("china.com");
                        } else {
                            mpostEntity.setG_asp("kdnet.net");
                            mpostEntity.setG_adp("kdnet.net");
                        }

                        //id
                        String id = "";
                        try {
//                            if (isTianYa == 1) {
//                                id = idge.getId("2", "M-BBS01-AI", "tianya.cn", "tianya.cn", g_ch_key);
//                            } else {
//                                id = idge.getId("2", "M-BBS01-AI", "kdnet.net", "kdnet.net", g_ch_key);
//                            }
                            if (isTianYa == 1) {
                                id = idge.getId("2", "M-BBS08-AI", "tianya.cn", "tianya.cn", g_ch_key);
                            } else if (isTianYa == 2) {//人民网强国论坛
                                id = idge.getId("2", "M-BBS08-AI", "people.com.cn", "people.com.cn", g_ch_key);
                            } else if (isTianYa == 3) {//爱卡
                                id = idge.getId("2", "M-BBS08-AI", "xcar.com.cn", "xcar.com.cn", g_ch_key);
                            } else if (isTianYa == 4) {//铁血
                                id = idge.getId("2", "M-BBS08-AI", "tiexue.net", "tiexue.net", g_ch_key);
                            } else if (isTianYa == 5) {//中华
                                id = idge.getId("2", "M-BBS08-AI", "china.com", "china.com", g_ch_key);
                            } else {
                                id = idge.getId("2", "M-BBS08-AI", "kdnet.net", "kdnet.net", g_ch_key);
                            }
                            mpostEntity.setG_id(id);
                        } catch (Exception e) {
                            logger.error("(g_id)Get id error,cause " + e.getMessage(), e);
                            logger.warn("@@g_ch_key:" + g_ch_key);
                            continue;
                        }

                        mpostEntity.setM_is_pt_generated(0);

                        long gt = System.currentTimeMillis() / 1000;
                        mpostEntity.setM_gather_time(gt);

                        String m_publish_time = dataMap.get("UPLOADTIME");
                        if (m_publish_time == null || m_publish_time.isEmpty() || "null".equals(m_publish_time)) {
                            //如果pt为空，则填gt
                            mpostEntity.setM_publish_time(gt);
                            mpostEntity.setM_is_pt_generated(1);

                            mpostEntity.setM_year(-1);
                            mpostEntity.setM_month(-1);
                            mpostEntity.setM_day(-1);
                            mpostEntity.setM_hour(-1);
                        } else {
                            try {
                                long str = TransformUtil.timeToLong(m_publish_time);
                                mpostEntity.setM_publish_time(str);
                            } catch (Exception e) {
                                logger.error("(m_publish_time)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_publish_time:" + m_publish_time);
                                mpostEntity.setM_publish_time(-1);
                                mpostEntity.setM_is_pt_generated(1);
                            }
                            try {
                                SimpleDateFormat sdfd = new SimpleDateFormat("yyyyMMddHHmmss");
                                Date parse = sdfd.parse(m_publish_time);
                                SimpleDateFormat sd = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
                                String time = sd.format(parse);
                                mpostEntity.setM_year(Long.parseLong(time.split(" ")[0].split("/")[0]));
                                mpostEntity.setM_month(Long.parseLong(time.split(" ")[0].split("/")[1]));
                                mpostEntity.setM_day(Long.parseLong(time.split(" ")[0].split("/")[2]));

                                mpostEntity.setM_hour(Long.parseLong(time.split(" ")[1].split(":")[0]));
                            } catch (Exception e) {
                                logger.error("(m_publish_time)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_publish_time:" + m_publish_time);
                                mpostEntity.setM_year(-1);
                                mpostEntity.setM_month(-1);
                                mpostEntity.setM_day(-1);
                                mpostEntity.setM_hour(-1);
                            }
                        }

                        mpostEntity.setM_insert_time(System.currentTimeMillis() / 1000);
                        //目前忽略，填默认值
                        mpostEntity.setM_update_time(-1);

                        String m_forms = dataMap.get("TYPE");
                        String[] forms = new String[]{};
                        if (m_forms == null || m_forms.isEmpty() || m_forms.equals("[]") || "null".equals(m_forms)) {
                            mpostEntity.setM_forms(forms);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(m_forms);
                                forms = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    forms[k] = jsonArray.getString(k);
                                }
                                mpostEntity.setM_forms(forms);
                            } catch (Exception e) {
                                logger.error("(m_forms)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_forms:" + m_forms);
                                mpostEntity.setM_forms(new String[]{});
                            }
                        }

                        String m_properties = dataMap.get("POSTTYPE");
                        String[] properties = new String[]{};
                        if (m_properties == null || m_properties.isEmpty() || m_properties.equals("[]") || "null".equals(m_properties)) {
                            mpostEntity.setM_properties(properties);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(m_properties);
                                properties = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    properties[k] = jsonArray.getString(k);
                                }
                                mpostEntity.setM_properties(properties);
                            } catch (Exception e) {
                                logger.error("(m_properties)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@,m_properties:" + m_properties);
                                mpostEntity.setM_properties(new String[]{});
                            }
                        }

                        //m_title_sim_tag,m_cont_sim_rank请求去重接口
                        String title = dataMap.get("TITLE");
                        String requestTitle = MD5Util.getMD5ByString("lt");
                        if (title == null || title.isEmpty() || "null".equals(title)) {
                            mpostEntity.setM_title("");
                            mpostEntity.setM_title_sim_tag("");
                        } else {
                            try {
                                title = new String(Base64.decodeFast(title), "UTF-8");
                                mpostEntity.setM_title(title);
                                String resCmdTitle = MD5Util.getMD5ByString(title);
                                requestTitle = MD5Util.getMD5ByString("lt" + title);;
                                mpostEntity.setM_title_sim_tag(resCmdTitle);
                            } catch (Exception e) {
                                logger.error("(m_title)decode base64 failed, cause " + e.getMessage(), e);
                                logger.warn("@@,m_title: " + dataMap.get("TITLE"));
                                mpostEntity.setM_title("");
                                mpostEntity.setM_title_sim_tag("");
                            }
                        }

                        String m_content = dataMap.get("TEXT");
                        String requestContent = MD5Util.getMD5ByString("lt");
                        if (m_content == null || m_content.isEmpty() || "null".equals(m_content)) {
                            mpostEntity.setM_content("");
                            mpostEntity.setM_cont_sim_tag("");
                            logger.warn("the m_content is empty,@@m_content:" + m_content);
                        } else {
                            try {
                                m_content = new String(Base64.decodeFast(m_content), "UTF-8");
                                mpostEntity.setM_content(m_content);
                                String resCmdTitle = MD5Util.getMD5ByString(m_content);
                                requestContent = MD5Util.getMD5ByString("lt" + m_content);
                                mpostEntity.setM_cont_sim_tag(resCmdTitle);
                            } catch (Exception e) {
                                logger.error("(m_content)decode base64 failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_content:" + dataMap.get("TEXT"));
                                mpostEntity.setM_content("");
                                mpostEntity.setM_cont_sim_tag("");
                            }
                        }

                        String m_html_content = dataMap.get("ORIGINPAGE");
                        if (m_html_content == null || m_html_content.isEmpty() || "null".equals(m_html_content)) {
                            mpostEntity.setM_html_content("");
                        } else {
                            try {
                                m_html_content = new String(Base64.decodeFast(m_html_content), "UTF-8");
                                mpostEntity.setM_html_content(m_html_content);
                            } catch (Exception e) {
                                logger.error("(m_html_content)decode base64 failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_html_content:" + dataMap.get("ORIGINPAGE"));
                                mpostEntity.setM_html_content("");
                            }
                        }

                        String m_emoticons = dataMap.get("EMOTION");
                        String[] emotions = new String[]{};
                        if (m_emoticons == null || m_emoticons.isEmpty() || m_emoticons.equals("[]") || "null".equals(m_emoticons)) {
                            mpostEntity.setM_emoticon(emotions);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(m_emoticons);
                                emotions = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String emotion = jsonArray.getString(k);
                                    emotions[k] = new String(Base64.decodeFast(emotion), "UTF-8");
                                }
                                mpostEntity.setM_emoticon(emotions);
                            } catch (Exception e) {
                                logger.error("decode base64 or (m_emoticon)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_emoticon:" + m_emoticons);
                                mpostEntity.setM_emoticon(new String[]{});
                            }
                        }

                        String langType = "";
                        try {
                            LangIdentifyTool langtool = new LangIdentifyTool();
                            langtool.initText(mpostEntity.getM_content());
                            langType = langtool.getLangType();
                            mpostEntity.setM_language(langType);
                        } catch (Exception e) {
                            logger.error("(m_language)get language error,cause " + e.getMessage(), e);
                            logger.warn("@@m_content:" + mpostEntity.getM_content());
                            logger.warn("the language return " + langType);
                            mpostEntity.setM_language("");
                        }

                        String m_domain_sign = dataMap.get("DOMAINSIGN");
                        if (m_domain_sign == null || m_domain_sign.isEmpty() || "null".equals(m_domain_sign)) {
                            mpostEntity.setM_domain_sign("");
                        } else {
                            mpostEntity.setM_domain_sign(m_domain_sign);
                        }

                        String m_mentioned_subjects = dataMap.get("TOPICS");
                        String[] subjects = new String[]{};
                        if (m_mentioned_subjects == null || m_mentioned_subjects.isEmpty() || m_mentioned_subjects.equals("[]") || "null".equals(m_mentioned_subjects)) {
                            mpostEntity.setM_mentioned_subjects(subjects);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(m_mentioned_subjects);
                                subjects = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String subject = jsonArray.getString(k);
                                    subjects[k] = new String(Base64.decodeFast(subject), "UTF-8");
                                }
                                mpostEntity.setM_mentioned_subjects(subjects);
                            } catch (Exception e) {
                                logger.error("decode base64 or (m_mentioned_subjects)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_mentioned_subjects:" + m_mentioned_subjects);
                                mpostEntity.setM_mentioned_subjects(new String[]{});
                            }
                        }

//                        if (isTianYa == 1) {
//                            mpostEntity.setM_site_id(60);
//                        } else {
//                            mpostEntity.setM_site_id(61);
//                        }
                        if (isTianYa == 1) {
                            mpostEntity.setM_site_id(60);
                        } else if (isTianYa == 2) {//人民网强国论坛
                            mpostEntity.setM_site_id(10000624728L);
                        } else if (isTianYa == 3) {//爱卡
                            mpostEntity.setM_site_id(10000193155L);
                        } else if (isTianYa == 4) {//铁血
                            mpostEntity.setM_site_id(5191704L);
                        } else if (isTianYa == 5) {//中华
                            mpostEntity.setM_site_id(7260495L);
                        } else {
                            mpostEntity.setM_site_id(61);
                        }
//                        mpostEntity.setM_site_location(110000);
//                        mpostEntity.setM_site_location(-1);

//                        if (isTianYa == 1) {
//                            long mSiteCategory = trs.getMSiteCategory("天涯论坛");
//                            logger.debug("m_site_category is : " + mSiteCategory);
//                            mpostEntity.setM_site_category(mSiteCategory + "");
//                        } else {
//                            long mSiteCategory = trs.getMSiteCategory("凯迪社区");
//                            logger.debug("m_site_category is : " + mSiteCategory);
//                            mpostEntity.setM_site_category(mSiteCategory + "");
//                        }
                        if (isTianYa == 1) {
                            long mSiteCategory = trs.getMSiteCategory("天涯论坛");
                            logger.debug("m_site_category is : " + mSiteCategory);
                            mpostEntity.setM_site_category(mSiteCategory + "");
                        } else if (isTianYa == 2) {//人民网强国论坛
                            long mSiteCategory = trs.getMSiteCategory("人民网强国论坛");
                            logger.debug("m_site_category is : " + mSiteCategory);
                            mpostEntity.setM_site_category(mSiteCategory + "");
                        } else if (isTianYa == 3) {//爱卡
//                            long mSiteCategory = trs.getMSiteCategory("爱卡汽车论坛");
//                            logger.debug("m_site_category is : " + mSiteCategory);
//                            mpostEntity.setM_site_category(mSiteCategory + "");
                            mpostEntity.setM_site_category("-1");
                        } else if (isTianYa == 4) {//铁血
                            long mSiteCategory = trs.getMSiteCategory("铁血论坛");
                            logger.debug("m_site_category is : " + mSiteCategory);
                            mpostEntity.setM_site_category(mSiteCategory + "");
                        } else if (isTianYa == 5) {//中华
                            long mSiteCategory = trs.getMSiteCategory("中华网论坛");
                            logger.debug("m_site_category is : " + mSiteCategory);
                            mpostEntity.setM_site_category(mSiteCategory + "");
                        } else if (isTianYa == 0) {//凯迪
                            long mSiteCategory = trs.getMSiteCategory("凯迪社区");
                            logger.debug("m_site_category is : " + mSiteCategory);
                            mpostEntity.setM_site_category(mSiteCategory + "");
                        }
                        String m_board_name = dataMap.get("COLUMN");
                        if (m_board_name == null || m_board_name.isEmpty() || "null".equals(m_board_name)) {
                            mpostEntity.setM_board_name("");
                        } else {
                            try {
                                m_board_name = new String(Base64.decodeFast(m_board_name), "UTF-8");
                                mpostEntity.setM_board_name(m_board_name);
                            } catch (Exception e) {
                                logger.error("(m_board_name)Decode base64 failed, cause " + e.getMessage(), e);
                                logger.warn("@@m_board_name:" + dataMap.get("COLUMN"));
                                mpostEntity.setM_board_name("");
                            }
                        }

                        long del = -1;
                        String isdel = dataMap.get("ISDEL");
                        if (isdel == null || isdel.isEmpty() || "null".equals(isdel)) {
                            del = -1;
                        } else {
                            try {
                                long str = Long.parseLong(isdel);
                                del = str;
                            } catch (Exception e) {
                                logger.error("(ISDEL)Type conversion failed, cause " + e.getMessage(), e);
                                logger.warn("@@ISDEL:" + isdel);
                                del = -1;
                            }
                        }
                        if (del == 0) {
                            //删除
                            mpostEntity.setMo_status(2);
                            String deltime = dataMap.get("DELTIME");
                            if (deltime == null || deltime.isEmpty() || "null".equals(deltime)) {
                                mpostEntity.setMo_time(-1);
                            } else {
                                try {
                                    long str = TransformUtil.timeToLong(deltime);
                                    mpostEntity.setMo_time(str);
                                } catch (Exception e) {
                                    logger.error("(DELTIME)Type conversion error,cause " + e.getMessage(), e);
                                    logger.warn("@@DELTIME:" + deltime);
                                    mpostEntity.setMo_time(-1);
                                }
                            }
                        } else if (del == 1) {
                            mpostEntity.setMo_status(0);
                            mpostEntity.setMo_time(-1);
                        } else {
                            mpostEntity.setMo_status(-1);
                            mpostEntity.setMo_time(-1);
                        }

                        String u_ch_id = dataMap.get("USERID");
                        if (u_ch_id == null || u_ch_id.isEmpty() || "null".equals(u_ch_id)) {
                            mpostEntity.setU_ch_id("");
                        } else {
                            mpostEntity.setU_ch_id(u_ch_id);
                        }
                        String u_g_ch_key = "";
                        try {
//                            if (isTianYa == 1) {
//                                //5.16做出修改
////                                                                u_g_ch_key = idge.getId("2", "U-BBS07-AI", "tianya.cn", "tianya.cn", mpostEntity.getU_ch_id());
//                                u_g_ch_key = "tianya.cn_" + mpostEntity.getU_ch_id();
//                            } else {
////                                                                u_g_ch_key = idge.getId("2", "U-BBS07-AI", "kdnet.net", "kdnet.net", mpostEntity.getU_ch_id());
//                                u_g_ch_key = "kdnet.net_" + mpostEntity.getU_ch_id();
//                            }
                            if (isTianYa == 1) {
                                u_g_ch_key = "tianya.cn_" + mpostEntity.getU_ch_id();
                            } else if (isTianYa == 2) {//人民网强国论坛
                                u_g_ch_key = "people.com.cn_" + mpostEntity.getU_ch_id();
                            } else if (isTianYa == 3) {//爱卡
                                u_g_ch_key = "xcar.com.cn_" + mpostEntity.getU_ch_id();
                            } else if (isTianYa == 4) {//铁血
                                u_g_ch_key = "tiexue.net_" + mpostEntity.getU_ch_id();
                            } else if (isTianYa == 5) {//中华
                                u_g_ch_key = "china.com_" + mpostEntity.getU_ch_id();
                            } else {
                                u_g_ch_key = "kdnet.net_" + mpostEntity.getU_ch_id();
                            }

                            mpostEntity.setU_g_ch_key(u_g_ch_key);
                        } catch (Exception e) {
                            logger.error("(u_gid)get id error,cause " + e.getMessage(), e);
                            logger.warn("@@u_gid:" + dataMap.get("USERID"));
                            mpostEntity.setU_g_ch_key("");
                        }

                        String u_send_ip = dataMap.get("IP");
                        if (u_send_ip == null || u_send_ip.isEmpty() || "null".equals(u_send_ip)) {
                            mpostEntity.setU_send_ip(-1);
                        } else {
                            try {
                                long str = TransformUtil.ipToLong(u_send_ip);
                                mpostEntity.setU_send_ip(str);
                            } catch (Exception e) {
                                logger.error("(u_send_ip)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@u_send_ip:" + u_send_ip);
                                mpostEntity.setU_send_ip(-1);
                            }
                        }

                        String m_reply_cnt = dataMap.get("POSTNUM");
                        if (m_reply_cnt == null || m_reply_cnt.isEmpty() || "null".equals(m_reply_cnt)) {
                            mpostEntity.setM_reply_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_reply_cnt);
                                mpostEntity.setM_reply_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_reply_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_reply_cnt:" + m_reply_cnt);
                                mpostEntity.setM_reply_cnt(-1);
                            }
                        }

                        String m_reply_user_cnt = dataMap.get("POSTPEOPLENUM");
                        if (m_reply_user_cnt == null || m_reply_user_cnt.isEmpty() || "null".equals(m_reply_user_cnt)) {
                            mpostEntity.setM_reply_user_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_reply_user_cnt);
                                mpostEntity.setM_reply_user_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_reply_user_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_reply_user_cnt:" + m_reply_user_cnt);
                                mpostEntity.setM_reply_user_cnt(-1);
                            }
                        }

                        String m_forward_cnt = dataMap.get("FORWARDNUM");
                        if (m_forward_cnt == null || m_forward_cnt.isEmpty() || "null".equals(m_forward_cnt)) {
                            mpostEntity.setM_forward_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_forward_cnt);
                                mpostEntity.setM_forward_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_forward_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_forward_cnt:" + m_forward_cnt);
                                mpostEntity.setM_forward_cnt(-1);
                            }
                        }

                        String m_like_cnt = dataMap.get("PRAISEDNUM");
                        if (m_like_cnt == null || m_like_cnt.isEmpty() || "null".equals(m_like_cnt)) {
                            mpostEntity.setM_like_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_like_cnt);
                                mpostEntity.setM_like_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_like_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_like_cnt:" + m_like_cnt);
                                mpostEntity.setM_like_cnt(-1);
                            }
                        }

                        String m_trampled_cnt = dataMap.get("TRAMPLEDNUM");
                        if (m_trampled_cnt == null || m_trampled_cnt.isEmpty() || "null".equals(m_trampled_cnt)) {
                            mpostEntity.setM_trampled_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_trampled_cnt);
                                mpostEntity.setM_trampled_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_trampled_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_trampled_cnt:" + m_trampled_cnt);
                                mpostEntity.setM_trampled_cnt(-1);
                            }
                        }

                        String m_sealed_cnt = dataMap.get("SEALEDNUM");
                        if (m_sealed_cnt == null || m_sealed_cnt.isEmpty() || "null".equals(m_sealed_cnt)) {
                            mpostEntity.setM_sealed_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_sealed_cnt);
                                mpostEntity.setM_sealed_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_sealed_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_sealed_cnt:" + m_sealed_cnt);
                                mpostEntity.setM_sealed_cnt(-1);
                            }
                        }

                        String m_rewarded_cnt = dataMap.get("REWARDEDNUM");
                        if (m_rewarded_cnt == null || m_rewarded_cnt.isEmpty() || "null".equals(m_rewarded_cnt)) {
                            mpostEntity.setM_rewarded_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_rewarded_cnt);
                                mpostEntity.setM_rewarded_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_rewarded_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_rewarded_cnt:" + m_rewarded_cnt);
                                mpostEntity.setM_rewarded_cnt(-1);
                            }
                        }

                        String m_send_cnt = dataMap.get("PUSHNUM");
                        if (m_send_cnt == null || m_send_cnt.isEmpty() || "null".equals(m_send_cnt)) {
                            mpostEntity.setM_send_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_send_cnt);
                                mpostEntity.setM_send_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_send_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_send_cnt:" + m_send_cnt);
                                mpostEntity.setM_send_cnt(-1);
                            }
                        }

                        String m_pv_cnt = dataMap.get("PVNUM");
                        if (m_pv_cnt == null || m_pv_cnt.isEmpty() || "null".equals(m_pv_cnt)) {
                            mpostEntity.setM_pv_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_pv_cnt);
                                mpostEntity.setM_pv_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_pv_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_pv_cnt:" + m_pv_cnt);
                                mpostEntity.setM_pv_cnt(-1);
                            }
                        }

                        String m_uv_cnt = dataMap.get("UVNUM");
                        if (m_uv_cnt == null || m_uv_cnt.isEmpty() || "null".equals(m_uv_cnt)) {
                            mpostEntity.setM_uv_cnt(-1);
                        } else {
                            try {
                                long str = Long.parseLong(m_uv_cnt);
                                mpostEntity.setM_uv_cnt(str);
                            } catch (Exception e) {
                                logger.error("(m_uv_cnt)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_uv_cnt:" + m_uv_cnt);
                                mpostEntity.setM_uv_cnt(-1);
                            }
                        }

                        mpostEntity.setMa_pic_ocr_contents(new String[]{});

                        String ma_picture_urls = dataMap.get("PICURLS");
                        String[] picture_urls = new String[]{};
                        String[] picture_files = new String[]{};
                        if (ma_picture_urls == null || ma_picture_urls.isEmpty() || ma_picture_urls.equals("[]") || "null".equals(ma_picture_urls)) {
                            mpostEntity.setMa_picture_urls(picture_urls);
                            mpostEntity.setMa_picture_cnt(-1);
                            mpostEntity.setMa_picture_files(picture_files);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(ma_picture_urls);
                                picture_urls = new String[jsonArray.size()];
                                picture_files = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String picture = jsonArray.getString(k);
                                    picture_urls[k] = new String(Base64.decodeFast(picture), "UTF-8");
                                    String endsignal = "";
                                    try {
                                        String str = picture_urls[k].substring(picture_urls[k].lastIndexOf("/") + 1, picture_urls[k].length());
                                        endsignal = str.substring(str.lastIndexOf("."), str.length());
                                    } catch (Exception e) {
                                        logger.warn(picture_urls[k] + " does not have endsignal.");
                                    }
                                    SimpleDateFormat sdfd = new SimpleDateFormat("yyyyMMdd");
                                    String currenttime = sdfd.format(new Date());
                                    String time3 = System.currentTimeMillis() / 1000 + "";
                                    String resCmd = MD5Util.getMD5ByString(picture_urls[k]);
                                    //bdtb/当天的日期/unix秒后三位/url的MD5.后缀
//                                    if (isTianYa == 1) {
//                                        picture_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    } else {
//                                        picture_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    }
                                    if (isTianYa == 1) {
                                        picture_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 2) {//人民网强国论坛
                                        picture_files[k] = "qglt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 3) {//爱卡
                                        picture_files[k] = "aklt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 4) {//铁血
                                        picture_files[k] = "txlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 5) {//中华
                                        picture_files[k] = "zhlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 0) {
                                        picture_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    }
                                }
                                mpostEntity.setMa_picture_urls(picture_urls);
                                mpostEntity.setMa_picture_cnt(jsonArray.size());
                                mpostEntity.setMa_picture_files(picture_files);
                            } catch (Exception e) {
                                logger.error("decode base64 or (ma_picture_urls)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@ma_picture_urls:" + ma_picture_urls);
                                mpostEntity.setMa_picture_urls(new String[]{});
                                mpostEntity.setMa_picture_cnt(-1);
                                mpostEntity.setMa_picture_files(picture_files);
                            }
                        }

                        String ma_video_urls = dataMap.get("VIDEOURLS");
                        String[] videos = new String[]{};
                        String[] videos_files = new String[]{};
                        if (ma_video_urls == null || ma_video_urls.isEmpty() || ma_video_urls.equals("[]") || "null".equals(ma_video_urls)) {
                            mpostEntity.setMa_video_urls(videos);
                            mpostEntity.setMa_video_cnt(-1);
                            mpostEntity.setMa_video_files(videos_files);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(ma_video_urls);
                                videos = new String[jsonArray.size()];
                                videos_files = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String video = jsonArray.getString(k);
                                    videos[k] = new String(Base64.decodeFast(video), "UTF-8");
                                    String endsignal = "";
                                    try {
                                        String str = videos[k].substring(videos[k].lastIndexOf("/") + 1, videos[k].length());
                                        endsignal = str.substring(str.lastIndexOf("."), str.length());
                                    } catch (Exception e) {
                                        logger.warn(videos[k] + " does not have endsignal.");
                                    }
                                    SimpleDateFormat sdfd = new SimpleDateFormat("yyyyMMdd");
                                    String currenttime = sdfd.format(new Date());
                                    String time3 = System.currentTimeMillis() / 1000 + "";
                                    String resCmd = MD5Util.getMD5ByString(videos[k]);
//                                    if (isTianYa == 1) {
//                                        videos_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    } else {
//                                        videos_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    }
                                    if (isTianYa == 1) {
                                        videos_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 2) {//人民网强国论坛
                                        videos_files[k] = "qglt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 3) {//爱卡
                                        videos_files[k] = "aklt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 4) {//铁血
                                        videos_files[k] = "txlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 5) {//中华
                                        videos_files[k] = "zhlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 0) {
                                        videos_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    }
                                }
                                mpostEntity.setMa_video_urls(videos);
                                mpostEntity.setMa_video_cnt(jsonArray.size());
                                mpostEntity.setMa_video_files(videos_files);
                            } catch (Exception e) {
                                logger.error("decode base64 or (ma_video_urls)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@ma_video_urls:" + ma_video_urls);
                                mpostEntity.setMa_video_urls(new String[]{});
                                mpostEntity.setMa_video_cnt(-1);
                                mpostEntity.setMa_video_files(new String[]{});
                            }
                        }

                        String ma_audio_urls = dataMap.get("AUDIORLS");
                        String[] audios = new String[]{};
                        String[] audios_files = new String[]{};
                        if (ma_audio_urls == null || ma_audio_urls.isEmpty() || ma_audio_urls.equals("[]") || "null".equals(ma_audio_urls)) {
                            mpostEntity.setMa_audio_urls(audios);
                            mpostEntity.setMa_audio_cnt(-1);
                            mpostEntity.setMa_audio_files(audios_files);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(ma_audio_urls);
                                audios = new String[jsonArray.size()];
                                audios_files = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String audio = jsonArray.getString(k);
                                    audios[k] = new String(Base64.decodeFast(audio), "UTF-8");
                                    String endsignal = "";
                                    try {
                                        String str = audios[k].substring(audios[k].lastIndexOf("/") + 1, audios[k].length());
                                        endsignal = str.substring(str.lastIndexOf("."), str.length());
                                    } catch (Exception e) {
                                        logger.warn(audios[k] + " does not have endsignal.");
                                    }
                                    SimpleDateFormat sdfd = new SimpleDateFormat("yyyyMMdd");
                                    String currenttime = sdfd.format(new Date());
                                    String time3 = System.currentTimeMillis() / 1000 + "";
                                    String resCmd = MD5Util.getMD5ByString(audios[k]);
//                                    if (isTianYa == 1) {
//                                        audios_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    } else {
//                                        audios_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    }
                                    if (isTianYa == 1) {
                                        audios_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 2) {//人民网强国论坛
                                        audios_files[k] = "qglt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 3) {//爱卡
                                        audios_files[k] = "aklt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 4) {//铁血
                                        audios_files[k] = "txlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 5) {//中华
                                        audios_files[k] = "zhlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 0) {
                                        audios_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    }
                                }
                                mpostEntity.setMa_audio_urls(audios);
                                mpostEntity.setMa_audio_cnt(jsonArray.size());
                                mpostEntity.setMa_audio_files(audios_files);
                            } catch (Exception e) {
                                logger.error("decode base64 or (ma_audio_urls)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@ma_audio_urls:" + ma_audio_urls);
                                mpostEntity.setMa_audio_urls(new String[]{});
                                mpostEntity.setMa_audio_cnt(-1);
                                mpostEntity.setMa_audio_files(new String[]{});
                            }
                        }

                        String ma_other_urls = dataMap.get("FILEURLS");
                        String[] other_urls = new String[]{};
                        String[] other_files = new String[]{};
                        if (ma_other_urls == null || ma_other_urls.isEmpty() || ma_other_urls.equals("[]") || "null".equals(ma_other_urls)) {
                            mpostEntity.setMa_other_urls(other_urls);
                            mpostEntity.setMa_picture_cnt(-1);
                            mpostEntity.setMa_other_files(other_files);
                        } else {
                            try {
                                JSONArray jsonArray = JSONArray.fromObject(ma_other_urls);
                                other_urls = new String[jsonArray.size()];
                                other_files = new String[jsonArray.size()];
                                for (int k = 0; k < jsonArray.size(); k++) {
                                    String other = jsonArray.getString(k);
                                    other_urls[k] = new String(Base64.decodeFast(other), "UTF-8");
                                    String endsignal = "";
                                    try {
                                        String str = other_urls[k].substring(other_urls[k].lastIndexOf("/") + 1, other_urls[k].length());
                                        endsignal = str.substring(str.lastIndexOf("."), str.length());
                                    } catch (Exception e) {
                                        logger.warn(other_urls[k] + " does not have endsignal.");
                                    }
                                    SimpleDateFormat sdfd = new SimpleDateFormat("yyyyMMdd");
                                    String currenttime = sdfd.format(new Date());
                                    String time3 = System.currentTimeMillis() / 1000 + "";
                                    String resCmd = MD5Util.getMD5ByString(other_urls[k]);
//                                    if (isTianYa == 1) {
//                                        other_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    } else {
//                                        other_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
//                                    }
                                    if (isTianYa == 1) {
                                        other_files[k] = "tylt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 2) {//人民网强国论坛
                                        other_files[k] = "qglt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 3) {//爱卡
                                        other_files[k] = "aklt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 4) {//铁血
                                        other_files[k] = "txlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 5) {//中华
                                        other_files[k] = "zhlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    } else if (isTianYa == 0) {
                                        other_files[k] = "kdlt/" + currenttime + "/" + time3.substring(time3.length() - 3, time3.length()) + "/" + resCmd + endsignal;
                                    }
                                }
                                mpostEntity.setMa_other_urls(other_urls);
                                mpostEntity.setMa_other_cnt(jsonArray.size());
                                mpostEntity.setMa_other_files(other_files);
                            } catch (Exception e) {
                                logger.error("decode base64 or (ma_other_urls)Field segmentation failed, cause " + e.getMessage(), e);
                                logger.warn("@@ma_other_urls:" + ma_other_urls);
                                mpostEntity.setMa_other_urls(new String[]{});
                                mpostEntity.setMa_other_cnt(-1);
                                mpostEntity.setMa_other_files(new String[]{});
                            }
                        }

                        if ("".equals(mpostEntity.getM_content())) {
                            mpostEntity.setM_keywords(new String[]{});
                            HashMap[] vmmap = new HashMap[1];
                            HashMap vm = new HashMap();
                            vm.put("v", "");
                            vm.put("w", "");
                            vmmap[0] = vm;
                            String vmmapStr = maps2json(vmmap);
                            mpostEntity.setM_keyword_vector(vmmapStr);

                            HashMap[] vmmap2 = new HashMap[1];
                            HashMap vm2 = new HashMap();
                            vm2.put("v", "");
                            vm2.put("w", "");
                            vmmap2[0] = vm2;
                            String vmmapStr2 = maps2json(vmmap2);
                            mpostEntity.setM_person_vector(vmmapStr2);
                            mpostEntity.setM_organization_vector(vmmapStr2);
                            mpostEntity.setM_region_vector(vmmapStr2);
                            mpostEntity.setM_country_vector(vmmapStr2);
                            mpostEntity.setM_province_vector(vmmapStr2);

                            mpostEntity.setM_regions(new String[]{});
                            mpostEntity.setM_countries(new String[]{});
                            mpostEntity.setM_provinces(new String[]{});
                            mpostEntity.setM_organizations(new String[]{});
                            mpostEntity.setM_persons(new String[]{});

                            mpostEntity.setM_sentiment(-1);

                            mpostEntity.setM_importance(-1);

//                                                        mpostEntity.setM_category_tag(new String[]{});
                            mpostEntity.setM_category_industry(new String[]{});
                        } else {
                            String[] keywords = trs.getMKeyWordList(mpostEntity.getM_content());
                            mpostEntity.setM_keywords(keywords);

                            //调用trs接口返回一个字符串，然后将这个字符串转换成structs类型的，然后再讲这个structs类型转换成json格式的字符串（map2json方法来实现）
                            HashMap[] keywordsmap = trs.getMKeyWordVector(mpostEntity.getM_content());
                            mpostEntity.setM_keyword_vector(maps2json(keywordsmap));

                            HashMap mxxVector = trs.getMXXVector(mpostEntity.getM_content());
                            for (Iterator columnsit = mxxVector.entrySet().iterator(); columnsit.hasNext();) {
                                Map.Entry<String, HashMap[]> entry = (Map.Entry<String, HashMap[]>) columnsit.next();
                                if ("person".equals(entry.getKey())) {
                                    mpostEntity.setM_person_vector(maps2json(entry.getValue()));
                                    String[] pers = new String[entry.getValue().length];
                                    for (int gv = 0; gv < entry.getValue().length; gv++) {
                                        pers[gv] = (String) entry.getValue()[gv].get("v");
                                    }
                                    mpostEntity.setM_persons(pers);
                                } else if ("organization".equals(entry.getKey())) {
                                    mpostEntity.setM_organization_vector(maps2json(entry.getValue()));
                                    String[] orgs = new String[entry.getValue().length];
                                    for (int gv = 0; gv < entry.getValue().length; gv++) {
                                        orgs[gv] = (String) entry.getValue()[gv].get("v");
                                    }
                                    mpostEntity.setM_organizations(orgs);
                                } else if ("region".equals(entry.getKey())) {
                                    mpostEntity.setM_region_vector(maps2json(entry.getValue()));
                                    mpostEntity.setM_country_vector(maps2json(entry.getValue()));
                                    mpostEntity.setM_province_vector(maps2json(entry.getValue()));
                                    String[] regs = new String[entry.getValue().length];
                                    for (int gv = 0; gv < entry.getValue().length; gv++) {
                                        regs[gv] = (String) entry.getValue()[gv].get("v");
                                    }
                                    mpostEntity.setM_regions(regs);
                                    mpostEntity.setM_countries(regs);
                                    mpostEntity.setM_provinces(regs);
                                }
                            }

                            long msentiment = trs.getMsentiment(mpostEntity.getM_content());
                            mpostEntity.setM_sentiment(msentiment);

                            //这个和下面 的几个调用trs的还没有改完，明天得问
//                                                        long mimportance = trs.getMImportance(mpostEntity.getM_content());
                            mpostEntity.setM_importance(-1);

//                                                        String[] mCategory_Tag = trs.getMCategory_Tag(mpostEntity.getM_content());
////                                                        mpostEntity.setM_category_tag(mCategory_Tag);
                            String[] mCatagoryIndustry = trs.getMCatagoryIndustry(mpostEntity.getM_content());
                            mpostEntity.setM_category_industry(mCatagoryIndustry);
                        }

//                                                
//                                                mpostEntity.setM_regions(new String[]{});
//                                                mpostEntity.setM_countries(new String[]{});
//                                                mpostEntity.setM_provinces(new String[]{});
//                                                mpostEntity.setM_organizations(new String[]{});
//                                                mpostEntity.setM_persons(new String[]{});
                        String m_snapshot_time = dataMap.get("SNAPSHOTTIME");
                        if (m_snapshot_time == null || m_snapshot_time.isEmpty() || "null".equals(m_snapshot_time)) {
                            mpostEntity.setM_snapshot_time(-1);
                        } else {
                            try {
                                long str = TransformUtil.timeToLong(m_snapshot_time);
                                mpostEntity.setM_snapshot_time(str);
                            } catch (Exception e) {
                                logger.error("(m_snapshot_time)Type conversion error,cause " + e.getMessage(), e);
                                logger.warn("@@m_snapshot_time:" + m_snapshot_time);
                                mpostEntity.setM_snapshot_time(-1);
                            }
                        }

                        String url_md5 = MD5Util.getMD5ByString(mpostEntity.getM_url());
                        mpostEntity.setM_url_md5(url_md5);
                        String u_name = searchmongodb.getValueByU_Name_Column(userMongoCollection, "_id", mpostEntity.getU_g_ch_key(), "u_name");
                        mpostEntity.setU_name(u_name);
                        count++;
                        requestString += id + "\n" + requestTitle + "\n" + requestContent + "\n";
                        mpostEntityList.add(mpostEntity);

                        //请求去重服务并发送数据
                        if (count % requestServerSize == 0 && count != 0) {
                            sendMessageToMQ(mpostEntityList, session, requestString);
                            requestString = "";
                            mpostEntityList.clear();
                        }
                    }//一条json数据
                }
            }//所有json数据
            if (!mpostEntityList.isEmpty()) {
                sendMessageToMQ(mpostEntityList, session, requestString);
                count = 0;
                requestString = "";
                mpostEntityList.clear();
            }
            logger.info("A total " + count + " data of the mpost.");
        } catch (Exception ex) {
            logger.error("setSession error, cause " + ex.getMessage(), ex);
        } finally {
            if (session != null) {
                pool.returnSession(session);
            }
        }
    }

    public void sendMessageToMQ(List<MPostEntity> mpostEntityList, RESendSession session, String requestString) {
        //请求去重服务返回的信息数组
        String[] returnIds = new String[]{};
        long delcount = 0;
        List<MPostEntity> realList = new ArrayList<MPostEntity>();
        int[] location =null;
        try {
            logger.info("mpostEntityList.size= " + mpostEntityList.size());
            logger.debug("requestString= " + requestString);
            returnIds = DeDuplication.deduplication(requestString);
        } catch (Exception e) {
            logger.error("request to de-duplication failed, cause " + e.getMessage(), e);
            return;
        }

        //去重部分
        //删除重复数据
        int k = 0;
        String yunrunJSONString = "";
        logger.info("dddddd" + returnIds.length);
        for (int s = 0; s < returnIds.length; s = s + 3) {
            if ("1".equals(returnIds[s])) {
                delcount++;
                logger.debug("remove one repeat data." + returnIds[s]);
                mpostEntityList.get(k).setM_title_sim_rank(-1);
                mpostEntityList.get(k).setM_cont_sim_rank(-1);
            } else {
                logger.debug("add" + returnIds[s]);
                realList.add(mpostEntityList.get(k));
                if ("1".equals(returnIds[s + 1])) {
                    //重复
                    mpostEntityList.get(k).setM_title_sim_rank(10000);
                } else {
                    mpostEntityList.get(k).setM_title_sim_rank(1000);
                }

                if ("1".equals(returnIds[s + 2])) {
                    //重复
                    mpostEntityList.get(k).setM_cont_sim_rank(10000);
                } else {
                    mpostEntityList.get(k).setM_cont_sim_rank(1000);
                }

                if (isTianYa == 1) {
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"tianya.cn\"},";
                } else if (isTianYa == 2) {//人民网强国论坛
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"people.com.cn\"},";
                } else if (isTianYa == 3) {//爱卡
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"xcar.com.cn\"},";
                } else if (isTianYa == 4) {//铁血
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"tiexue.net\"},";
                } else if (isTianYa == 5) {//中华
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"china.com\"},";
                } else if (isTianYa == 0) {
                    yunrunJSONString += "{" + "\"m_board_name\":\"" + mpostEntityList.get(k).getM_board_name() + "\",\"g_asp\":\"kdnet.net\"},";
                }
            }
            k++;
        }
        //去重数据条数
        logger.info("remove datas of the mpost: " + delcount);
        logger.debug("yunrunJSONString   ----- >" + yunrunJSONString);
        if (!yunrunJSONString.equals("")) {
            yunrunJSONString = "[" + yunrunJSONString.substring(0, yunrunJSONString.length() - 1) + "]";

            HashMap yunmap = new HashMap();
            yunmap.put("articlelist", yunrunJSONString);
            location = new MSiteLocation().send(yunrunURL, yunmap);
            if (location == null || location.length != realList.size()) {
                logger.error("the yunrun return null or the message.size != realList.size! And the location size is " + location.length + " and the realList size is " + realList.size());
                location = new int[realList.size()];
                for (int lo = 0; lo < realList.size(); lo++) {
                    location[lo] = 0;
                }
            }
        }
        //循环发送数据
        try {
            //将realList更换成mpostEntityList了，这样的数据就不是去重之后的数据了，到时候更改回来即可
            //20170426 已经修改回来了
            for (int b = 0; b < realList.size(); b++) {

                MPostEntity entity = realList.get(b);
                if (entity != null) {
                    session.setString("g_id", entity.getG_id());
//                                        session.setLong("g_channel", entity.getG_channel());
//                                        session.setString("g_spec", entity.getG_spec());
                    session.setString("g_ch_key", entity.getG_ch_key());
                    session.setString("g_asp", entity.getG_asp());
                    session.setString("g_adp", entity.getG_adp());
                    session.setLong("m_is_pt_generated", entity.getM_is_pt_generated());
                    session.setLong("m_publish_time", entity.getM_publish_time());
                    session.setLong("m_insert_time", entity.getM_insert_time());
                    session.setString("m_url", entity.getM_url());
                    session.setStrings("m_forms", entity.getM_forms());
                    session.setStrings("m_properties", entity.getM_properties());
                    session.setString("m_title", entity.getM_title());
                    session.setString("m_title_sim_tag", entity.getM_title_sim_tag());
                    session.setLong("m_title_sim_rank", entity.getM_title_sim_rank());
                    session.setString("m_content", entity.getM_content());
                    session.setString("m_cont_sim_tag", entity.getM_cont_sim_tag());
                    session.setLong("m_cont_sim_rank", entity.getM_cont_sim_rank());
                    session.setString("m_html_content", entity.getM_html_content());
                    session.setStrings("m_emoticon", entity.getM_emoticon());
                    session.setString("m_language", entity.getM_language());
                    session.setString("m_domain_sign", entity.getM_domain_sign());
                    session.setStrings("m_mentioned_subjects", entity.getM_mentioned_subjects());
                    session.setLong("m_site_id", entity.getM_site_id());
                    session.setLong("m_site_location", location[b]);
                    session.setString("m_site_category", entity.getM_site_category());
                    session.setString("m_board_name", entity.getM_board_name());
                    session.setLong("mo_status", entity.getMo_status());
                    session.setLong("mo_time", entity.getMo_time());
                    session.setString("u_g_ch_key", entity.getU_g_ch_key());
                    session.setString("u_ch_id", entity.getU_ch_id());
                    session.setLong("u_send_ip", entity.getU_send_ip());
                    session.setLong("u_access_type", entity.getU_access_type());
                    session.setLong("m_reply_cnt", entity.getM_reply_cnt());
                    session.setLong("m_reply_user_cnt", entity.getM_reply_user_cnt());
                    session.setLong("m_forward_cnt", entity.getM_forward_cnt());
                    session.setLong("m_like_cnt", entity.getM_like_cnt());
                    session.setLong("m_trampled_cnt", entity.getM_trampled_cnt());
                    session.setLong("m_sealed_cnt", entity.getM_sealed_cnt());
                    session.setLong("m_rewarded_cnt", entity.getM_rewarded_cnt());
                    session.setLong("m_send_cnt", entity.getM_send_cnt());
                    session.setLong("m_pv_cnt", entity.getM_pv_cnt());
                    session.setLong("m_uv_cnt", entity.getM_uv_cnt());
                    session.setLong("ma_picture_cnt", entity.getMa_picture_cnt());
                    session.setStrings("ma_picture_urls", entity.getMa_picture_urls());
                    session.setStrings("ma_picture_files", entity.getMa_picture_files());
                    session.setStrings("ma_pic_ocr_contents", entity.getMa_pic_ocr_contents());
                    session.setLong("ma_video_cnt", entity.getMa_video_cnt());
                    session.setStrings("ma_video_urls", entity.getMa_video_urls());
                    session.setStrings("ma_video_files", entity.getMa_video_files());
                    session.setLong("ma_audio_cnt", entity.getMa_audio_cnt());
                    session.setStrings("ma_audio_urls", entity.getMa_audio_urls());
                    session.setStrings("ma_audio_files", entity.getMa_audio_files());
                    session.setStrings("m_keywords", entity.getM_keywords());
                    session.setString("m_keyword_vector", entity.getM_keyword_vector());
                    session.setString("m_person_vector", entity.getM_person_vector());
                    session.setString("m_organization_vector", entity.getM_organization_vector());
                    session.setString("m_region_vector", entity.getM_region_vector());
                    session.setString("m_country_vector", entity.getM_country_vector());
                    session.setString("m_province_vector", entity.getM_province_vector());
                    session.setLong("m_sentiment", entity.getM_sentiment());
                    session.setLong("m_importance", entity.getM_importance());
//                                        session.setStrings("m_category_tag", entity.getM_category_tag());
                    session.setStrings("m_category_industry", entity.getM_category_industry());
                    session.setStrings("m_regions", entity.getM_regions());
                    session.setStrings("m_countries", entity.getM_countries());
                    session.setStrings("m_provinces", entity.getM_provinces());
                    session.setLong("m_snapshot_time", entity.getM_snapshot_time());
                    session.setLong("ma_other_cnt", entity.getMa_other_cnt());
                    session.setStrings("ma_other_urls", entity.getMa_other_urls());
                    session.setStrings("ma_other_files", entity.getMa_other_files());
                    session.setString("m_url_md5", entity.getM_url_md5());
//                                        session.setStrings("m_ymd", entity.getM_ymd());
                    session.setLong("m_year", entity.getM_year());
                    session.setLong("m_month", entity.getM_month());
                    session.setLong("m_day", entity.getM_day());
                    session.setLong("m_hour", entity.getM_hour());
                    session.setLong("m_gather_time", entity.getM_gather_time());
                    session.setStrings("m_organizations", entity.getM_organizations());
                    session.setStrings("m_persons", entity.getM_persons());
                    session.setLong("m_update_time", entity.getM_update_time());
                    session.setString("u_name", entity.getU_name());
                    session.setString("m_board_id", entity.getM_board_name());
                    try {
                        session.add();
                    } catch (Exception e) {
                        sendMSG(session);
                        session.add();
                    }

                    if (System.currentTimeMillis() / 1000 - starttime > timeout) {
                        starttime = System.currentTimeMillis() / 1000;
                        sendMSG(session);
                        logger.info("timeout.");
                    }
                }
            }
            sendMSG(session);
            logger.info("send datas of mpost: " + realList.size());
        } catch (Exception ex) {
            logger.error("setSession error, cause " + ex.getMessage(), ex);
        }
    }

    public void sendMSG(RESendSession session) {
        while (true) {
            try {
                logger.info("flush...");
                session.flush();
                break;
            } catch (Exception ex) {
                logger.info("what's up buddy ? how's it going ?" + ex, ex);
            }
        }
    }

    public String maps2json(Map[] maps) {
        String jsons = "";
        for (Map map : maps) {
            String json = JSONObject.toJSONString(map);
            jsons += (json + ",");
        }
        if (maps.length > 0) {
            jsons = jsons.substring(0, jsons.lastIndexOf(","));
        }
        jsons = "[" + jsons + "]";
        return jsons;
    }
}
